{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Exploratory Data Analysis (EDA) notebook\n",
    "\n",
    "### Here we explore the provided dataset and we analyze some of it's statistical characteristics"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0ddd434",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.590791Z",
     "end_time": "2023-07-30T22:40:22.634792Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import copy\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "import seaborn as sns\n",
    "import colorcet as cc\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "483b632a",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.603790Z",
     "end_time": "2023-07-30T22:40:22.669793Z"
    }
   },
   "outputs": [],
   "source": [
    "sns.set_style('darkgrid')\n",
    "sns.set(rc={'figure.figsize':(14,8)})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20ee4677",
   "metadata": {},
   "source": [
    "### Loading datasets as Panda Dataframes from the correct path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2eea2068",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.619791Z",
     "end_time": "2023-07-30T22:40:22.669793Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_names = [\"ElBorn.csv\", \"LesCorts.csv\", \"PobleSec.csv\"]\n",
    "data_path = \"../dataset\"\n",
    "datasets = [os.path.join(data_path, data_name) for data_name in dataset_names]\n",
    "\n",
    "print(datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fafd794",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.634792Z",
     "end_time": "2023-07-30T22:40:22.724791Z"
    }
   },
   "outputs": [],
   "source": [
    "dfs = dict()\n",
    "for data_name, data_path in zip(dataset_names, datasets):\n",
    "    dfs[data_name] = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad96eadc",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.700791Z",
     "end_time": "2023-07-30T22:40:22.739791Z"
    }
   },
   "outputs": [],
   "source": [
    "elborn = dfs[\"ElBorn.csv\"]\n",
    "lescorts = dfs[\"LesCorts.csv\"]\n",
    "poblesec = dfs[\"PobleSec.csv\"]\n",
    "elborn[\"time\"] = pd.to_datetime(elborn[\"time\"], format=\"%Y-%m-%d %H:%M:%S\")\n",
    "lescorts[\"time\"] = pd.to_datetime(lescorts[\"time\"], format=\"%Y-%m-%d %H:%M:%S\")\n",
    "poblesec[\"time\"] = pd.to_datetime(poblesec[\"time\"], format=\"%Y-%m-%d %H:%M:%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65677813",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.729797Z",
     "end_time": "2023-07-30T22:40:22.770792Z"
    }
   },
   "outputs": [],
   "source": [
    "print(f\"Elborn: {min(elborn['time'])}, {max(elborn['time'])}\")\n",
    "print(f\"LesCorts: {min(lescorts['time'])}, {max(lescorts['time'])}\")\n",
    "print(f\"PobleSec: {min(poblesec['time'])}, {max(poblesec['time'])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "037ba315",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.758792Z",
     "end_time": "2023-07-30T22:40:22.774791Z"
    }
   },
   "outputs": [],
   "source": [
    "elborn.set_index(\"time\", inplace=True)\n",
    "lescorts.set_index(\"time\", inplace=True)\n",
    "poblesec.set_index(\"time\", inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c802fe91",
   "metadata": {},
   "source": [
    "### The size and data form of each individual base station dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "483d0086",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.778795Z",
     "end_time": "2023-07-30T22:40:22.792794Z"
    }
   },
   "outputs": [],
   "source": [
    "len(elborn), len(lescorts), len(poblesec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8683c2d",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.791792Z",
     "end_time": "2023-07-30T22:40:22.836793Z"
    }
   },
   "outputs": [],
   "source": [
    "elborn.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cb1e1e1",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.815793Z",
     "end_time": "2023-07-30T22:40:22.853793Z"
    }
   },
   "outputs": [],
   "source": [
    "lescorts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db164960",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.830793Z",
     "end_time": "2023-07-30T22:40:22.904805Z"
    }
   },
   "outputs": [],
   "source": [
    "poblesec.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d2dbeaa",
   "metadata": {},
   "source": [
    "### Up and Down Limits for each dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1206138f",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.840793Z",
     "end_time": "2023-07-30T22:40:22.905805Z"
    }
   },
   "outputs": [],
   "source": [
    "elborn.up.min(), elborn.down.min(),lescorts.up.min(), lescorts.down.min(),poblesec.up.min(), poblesec.down.min(),"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25569157",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.854793Z",
     "end_time": "2023-07-30T22:40:22.989804Z"
    }
   },
   "outputs": [],
   "source": [
    "elborn.up.max(), elborn.down.max(),lescorts.up.max(), lescorts.down.max(),poblesec.up.max(), poblesec.down.max(),"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e0cc5c4",
   "metadata": {},
   "source": [
    "### Correlation Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5489e881",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.870793Z",
     "end_time": "2023-07-30T22:40:22.990805Z"
    }
   },
   "outputs": [],
   "source": [
    "def cor_matrix(df):\n",
    "    plt.figure(figsize=(18,8))\n",
    "    sns.heatmap(df.corr(),annot=True,cmap='Greens',linewidths=0.2)\n",
    "    plt.show()\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fdffe23",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:22.886806Z",
     "end_time": "2023-07-30T22:40:23.922476Z"
    }
   },
   "outputs": [],
   "source": [
    "cor_matrix(elborn)\n",
    "cor_matrix(lescorts)\n",
    "cor_matrix(poblesec)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f0965f0",
   "metadata": {},
   "source": [
    "### 10 points rolling mean for every base station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d28e797",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-07-30T22:40:23.935073Z",
     "end_time": "2023-07-30T22:40:24.088082Z"
    }
   },
   "outputs": [],
   "source": [
    "elborn_up_mean = elborn[\"up\"]\n",
    "plt.ticklabel_format(style='plain')\n",
    "ax = elborn_up_mean.plot(figsize=(8,6))\n",
    "ax.set_xlabel(\"Date\")\n",
    "ax.set_title(\"10 points rolling mean\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c6b3781",
   "metadata": {},
   "outputs": [],
   "source": [
    "elborn_down_mean = elborn[\"down\"].rolling(window=10).mean()\n",
    "plt.ticklabel_format(style='plain')\n",
    "ax = elborn_up_mean.plot(figsize=(8,6))\n",
    "ax.set_xlabel(\"Date\")\n",
    "ax.set_title(\"10 points rolling mean\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a144819f",
   "metadata": {},
   "source": [
    "### Seasonal Box Plot\n",
    "Boxplot is a method for graphically demonstrating the locality, spread and skewness groups of numerical data through their quartiles. We create boxplots for every hour in order to examine how every base station statistically correlates with each other."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "567f53d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def seasonal_box_plot(df, column):\n",
    "    tmp = copy.deepcopy(df)\n",
    "    tmp[\"day\"] = tmp.index.dayofyear\n",
    "    tmp[\"hour\"] = tmp.index.hour\n",
    "    fig, ax = plt.subplots(figsize=(8, 6))\n",
    "    ax.ticklabel_format(style='plain')\n",
    "    \n",
    "    palette = sns.color_palette(cc.glasbey, n_colors=tmp.day.nunique())\n",
    "    \n",
    "    sns.boxplot(x=tmp['hour'], y=tmp[column])\n",
    "    ax.set_title('', fontsize = 20, loc='center', fontdict=dict(weight='bold'))\n",
    "    ax.set_xlabel('Hour', fontsize = 16, fontdict=dict(weight='bold'))\n",
    "    ax.set_ylabel('', fontsize = 16, fontdict=dict(weight='bold'))\n",
    "    plt.show()\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c42fa4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "elborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3d594db",
   "metadata": {},
   "outputs": [],
   "source": [
    "seasonal_box_plot(elborn, \"up\")\n",
    "seasonal_box_plot(elborn, \"down\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "496bc9f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "seasonal_box_plot(lescorts, \"up\")\n",
    "seasonal_box_plot(lescorts, \"down\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f3fdefb",
   "metadata": {},
   "outputs": [],
   "source": [
    "seasonal_box_plot(poblesec, \"up\")\n",
    "seasonal_box_plot(poblesec, \"down\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27cec539",
   "metadata": {},
   "source": [
    "### Concatenate datasets from every base station together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62bb13ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "elborn[\"District\"] = \"ElBorn\"\n",
    "lescorts[\"District\"] = \"LesCorts\"\n",
    "poblesec[\"District\"] = \"PobleSec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fbf8f7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df = pd.concat([elborn, lescorts, poblesec], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c254c5de",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.groupby(['District']).min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9166048",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.groupby(['District']).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "174c81f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.groupby(['District']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb4940a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.groupby(['District']).median()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "a39106e1a9d6d153b7400628e7589ff266b5caee5b0db427f0903be982155882"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
